# # coding: utf-8
# import os
# import imp
# import json
# import hashlib
# import datetime
# import uuid
# import time
# import shutil
# import functools
# import subprocess
# import traceback
# import inspect
# import multiprocessing
# import logging as log
# # from celery import Celery, platforms
#
# # cele = Celery('run_code', broker='redis://192.168.18.21:6379/3',)
# #
# # platforms.C_FORCE_ROOT = True
#
#
# from kombu import serialization
# serialization.registry._decoders.pop("application/x-python-serialize")
# # cele.conf.update(
# #     CELERY_ACCEPT_CONTENT = ['json'],
# #     CELERY_TASKS_SERIALIZER = 'json',
# #     CELERY_RESULT_SERIALIZER = 'json',
# # )
#
#
# # 待测试配置文件临时存放目录
# cur_path = os.path.dirname(os.path.abspath(__file__))
# temp_test_dir = os.path.join(cur_path, "temp_test_config")
# if not os.path.exists(temp_test_dir):
#     os.mkdir(temp_test_dir)
#
# def handle_func(e, func):
#     pass
#     # log.logger.exception(e)
#     return
#
#
# def keepalive(handle_func=None, interval=1):
#     '''装饰器
#     功能：
#        捕获被装饰函数的异常并重新调用函数
#        函数正常结束则结束
#     装饰器参数：
#        @handle_func:function
#           异常处理函数 默认接收参数 e(异常对象), func(被装饰函数)
#        @interval:number
#           函数重启间隔
#     '''
#     def wrapper(func):
#         @functools.wraps(func)
#         def keep(*args, **kwargs):
#             while 1:
#                 try:
#                     result = func(*args, **kwargs)
#                 except Exception as e:
#                     if handle_func:
#                         handle_func(e, func)
#                     time.sleep(interval)
#                     continue
#                 break
#             return result
#         return keep
#     return wrapper
#
#
# # @keepalive(handle_func=handle_func)
# # @cele.task
# # def step_debug_process(task_queue):
# #     try:
# #         try:
# #             task, typ, proxy, url, next_func = task_queue
# #         except:
# #             time.sleep(1)
# #             return
# #         if typ not in [2, ]:
# #             return
# #         try:
# #             url = json.loads(url, cls=DateTimeDecoder)
# #         except Exception as e:
# #             pass
# #         # 计时
# #         _start = time.time()
# #         _warn_time_list = [120, 300, 600]
# #         #
# #         config_file = os.path.join(temp_test_dir, "%s.py" % task)
# #         kill_file = os.path.join(temp_test_dir, "%s.kill" % task)
# #         result_file = os.path.join(temp_test_dir, "%s.result" % task)
# #         status_file = os.path.join(temp_test_dir, "%s.status" % task)
# #
# #         # 执行状态记录
# #         with open(status_file, "w") as f:
# #             f.write("0")
# #         result = {}
# #         with open(config_file) as f:
# #             code = f.read()
# #
# #         is_ok = 1
# #         module = imp.new_module("spider")
# #         try:
# #             exec (code in module.__dict__)
# #         except Exception as e:
# #             result = {"code": 1, "error": traceback.format_exc()}
# #             is_ok = 0
# #         if is_ok:
# #             try:
# #                 myspider = module.MySpider()
# #                 myspider.proxy_enable = True if proxy == 1 else False
# #                 myspider.init_dedup()
# #                 myspider.init_downloader()
# #
# #                 func_result = []
# #                 if next_func == 'get_start_urls':
# #                     func_result = myspider.get_start_urls({})
# #                 else:
# #                     if url in ['', 'None', 'null']:
# #                         url = None
# #                     resp = myspider.download(url)
# #                     func = getattr(myspider, next_func)
# #                     if isinstance(url, unicode):
# #                         url = url.encode("utf8")
# #                     if func:
# #                         if "url" in inspect.getargspec(func).args:
# #                             func_result = func(response=resp, url=url)
# #                         else:
# #                             func_result = func(response=resp)
# #                 result = {"code": 1, "data": [func_result]}
# #                 with open(result_file, "w") as f:
# #                     f.write(result)
# #             except Exception as e:
# #                 result = {"code": 1, "error": traceback.format_exc()}
# #                 is_ok = 0
# #         if is_ok:
# #             try:
# #                 if next_func == 'get_start_urls':
# #                     result = {"code": 0, "urls": [{"url": url, 'next': 'parse'} for url in func_result], "data": []}
# #                 elif next_func != 'parse_detail_page':
# #                     list_urls, callback, next_page_url = func_result
# #                     if callback:
# #                         callback = getattr(callback, "__name__", str(callback)).split(".")[-1]
# #                     else:
# #                         callback = 'parse_detail_page'
# #                     urls = [{'url': url, 'next': callback} for url in list_urls]
# #                     if next_page_url:
# #                         urls += [{'url': next_page_url, 'next': next_func, "is_next": 1}]
# #                     result = {"code": 0, "urls": urls, "data": []}
# #                 elif next_func == 'parse_detail_page':
# #                     if not func_result:
# #                         func_result = []
# #                     else:
# #                         if not isinstance(func_result, list):
# #                             func_result = [func_result]
# #                     result = {"code": 0, "data": [json.dumps(x, cls=DateTimeEncoder) for x in func_result],
# #                               "urls": []}
# #
# #                 with open(result_file, "w") as f:
# #                     f.write(result)
# #             except Exception as e:
# #                 result = {"code": 1, "error": traceback.format_exc()}
# #         # 清理
# #         os.remove(config_file)
# #         #
# #         result = json.dumps(result, cls=DateTimeEncoder)
# #         with open(result_file, "w") as f:
# #             f.write(result)
# #         # 执行状态记录
# #         with open(status_file, "w") as f:
# #             f.write("1")
# #     except KeyboardInterrupt:
# #         pass
# #     except Exception as e:
# #         print(e)
# #     return
#
# # @keepalive(handle_func=handle_func)
# @cele.task
# def webspider_test_process(task_queue):
#     try:
#         try:
#             task, typ, proxy, url, next_func = task_queue
#         except:
#             return
#         # if typ == 2:
#         #     step_debug_process(task, typ, proxy, url, next_func)
#         #计时
#         _start = time.time()
#         _warn_time_list = [10, 20,30]
#         #_warn_time_list = [10, 20, 30, 40, 50, 60]
#         #
#         config_file = os.path.join(temp_test_dir, "%s.py"%task)
#         kill_file = os.path.join(temp_test_dir, "%s.kill"%task)
#         result_file = os.path.join(temp_test_dir, "%s.result"%task)
#         status_file = os.path.join(temp_test_dir, "%s.status"%task)
#         # 执行状态记录
#         with open(status_file, "w") as f:
#             f.write("0")
#         #
#         temp_config_file = os.path.join("/work/spider", "%s.py" % task)
#         if typ == 0:
#             log.warning(temp_config_file)
#             shutil.copy(config_file, temp_config_file)
#             cmd = "docker run -v /work/spider:/work/spider --name %s 3aa027829065 python %s" % (task,temp_config_file)
#         elif typ == 1:
#             cmd = "docker run -v %s:%s --name %s test_spider:latest python /work/spider/run.py  --debug --config %s" % (temp_test_dir,temp_test_dir,task,config_file)
#             log.warning(cmd)
#             if proxy == 1:
#                 cmd += " --proxy"
#         result_file = open(result_file, "a+")
#         child_process = subprocess.Popen(cmd.split(), cwd="/work/spider", stdout=result_file, stderr=result_file)
#         # child_process = subprocess.Popen(cmd.split(), cwd="/work/lxt", stdout=result_file, stderr=result_file)
#         while 1:
#             returncode = child_process.poll()
#             if returncode is not None:
#                 break
#             if os.path.exists(kill_file):
#                 os.remove(kill_file)
#                 cmd = 'docker stop %s' % task
#                 child_process1 = subprocess.Popen(cmd.split(), cwd="/work/spider")
#                 # log.warning('aaa')
#                 child_process1.wait()
#                 # log.warning('bbb')
#                 child_process.kill()
#                 # child_process1.kill()
#                 os.kill(child_process.pid, 9)
#                 break
#             # 警告
#             exec_time = time.time() - _start
#             if exec_time > _warn_time_list[0]:
#                 _warn_time_list = _warn_time_list[1:]
#                 result_file.write("此脚本已执行 %s 秒\n" % exec_time)
#             if not _warn_time_list:
#                 result_file.write("此脚本执行超时 已停止")
#                 cmd = 'docker stop %s' % task
#                 child_process1 = subprocess.Popen(cmd.split(), cwd="/work/spider")
#                 log.warning('aaa')
#                 child_process1.wait()
#                 log.warning('bbb')
#                 child_process.kill()
#                 # child_process1.kill()
#                 os.kill(child_process.pid, 9)
#                 break
#             result_file.flush()
#             time.sleep(1)
#         #清理
#         os.remove(config_file)
#         if typ == 0:
#             os.remove(temp_config_file)
#         result_file.close()
#         # 执行状态记录
#         with open(status_file, "w") as f:
#             f.write("1")
#         del child_process
#     except KeyboardInterrupt:
#         try:
#             if typ == 0 and os.path.exists(temp_config_file):
#                 os.remove(temp_config_file)
#         except:
#             pass
#     return
